<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
  <title>Class: Robotstxt::Parser [Robotstxt]</title>
  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
  <meta http-equiv="Content-Script-Type" content="text/javascript" />
  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
  <script type="text/javascript">
  // <![CDATA[

  function popupCode( url ) {
    window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
  }

  function toggleCode( id ) {
    if ( document.getElementById )
      elem = document.getElementById( id );
    else if ( document.all )
      elem = eval( "document.all." + id );
    else
      return false;

    elemStyle = elem.style;

    if ( elemStyle.display != "block" ) {
      elemStyle.display = "block"
    } else {
      elemStyle.display = "none"
    }

    return true;
  }

  // Make codeblocks hidden by default
  document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }<\/style>" )

  // ]]>
  </script>

</head>
<body>


    <div id="classHeader">
        <table class="header-table">
        <tr class="top-aligned-row">
          <td><strong>Class</strong></td>
          <td class="class-name-in-header">Robotstxt::Parser</td>
        </tr>
        <tr class="top-aligned-row">
            <td><strong>In:</strong></td>
            <td>


                <a href="../../files/lib/robotstxt/parser_rb.html">

                lib/robotstxt/parser.rb

                </a>


        <br />

            </td>
        </tr>


        <tr class="top-aligned-row">
            <td><strong>Parent:</strong></td>
            <td>

                Object

            </td>
        </tr>

        </table>
    </div>
  <!-- banner header -->

  <div id="bodyContent">

  <div id="contextContent">

   </div>


    <div id="method-list">
      <h3 class="section-bar">Methods</h3>

      <div class="name-list">

        <a href="#M000005">allowed?</a>&nbsp;&nbsp;

        <a href="#M000007">found?</a>&nbsp;&nbsp;

        <a href="#M000004">get</a>&nbsp;&nbsp;

        <a href="#M000003">new</a>&nbsp;&nbsp;

        <a href="#M000006">sitemaps</a>&nbsp;&nbsp;

      </div>
    </div>

  </div>

    <!-- if includes -->

    <div id="section">



    <div id="attribute-list">
      <h3 class="section-bar">Attributes</h3>

      <div class="name-list">
        <table>

        <tr class="top-aligned-row context-row">
          <td class="context-item-name">body</td>

          <td class="context-item-value">&nbsp;[R]&nbsp;</td>

          <td class="context-item-desc"></td>
        </tr>

        <tr class="top-aligned-row context-row">
          <td class="context-item-name">found</td>

          <td class="context-item-value">&nbsp;[R]&nbsp;</td>

          <td class="context-item-desc"></td>
        </tr>

        <tr class="top-aligned-row context-row">
          <td class="context-item-name">robot_id</td>

          <td class="context-item-value">&nbsp;[RW]&nbsp;</td>

          <td class="context-item-desc"></td>
        </tr>

        <tr class="top-aligned-row context-row">
          <td class="context-item-name">rules</td>

          <td class="context-item-value">&nbsp;[R]&nbsp;</td>

          <td class="context-item-desc"></td>
        </tr>

        <tr class="top-aligned-row context-row">
          <td class="context-item-name">sitemaps</td>

          <td class="context-item-value">&nbsp;[R]&nbsp;</td>

          <td class="context-item-desc"></td>
        </tr>

        </table>
      </div>
    </div>


    <!-- if method_list -->

    <div id="methods">

      <h3 class="section-bar">Public Class methods</h3>


      <div id="method-M000003" class="method-detail">
        <a name="M000003"></a>

        <div class="method-heading">

          <a href="#M000003" class="method-signature">

          <span class="method-name">new</span><span class="method-args">(robot_id = nil)</span>

          </a>

        </div>

        <div class="method-description">

          <p>
Initializes a new Robots::Robotstxtistance with <tt>robot_id</tt> option.
</p>
<p>
<tt>client = Robotstxt::Robotstxtistance.new('my_robot_id')</tt>
</p>

          <p><a class="source-toggle" href="#"
            onclick="toggleCode('M000003-source');return false;">[Source]</a></p>
          <div class="method-source-code" id="M000003-source">
<pre>
    <span class="ruby-comment cmt"># File lib/robotstxt/parser.rb, line 29</span>
29:                 <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">initialize</span>(<span class="ruby-identifier">robot_id</span> = <span class="ruby-keyword kw">nil</span>)
30:                         
31:                         <span class="ruby-ivar">@robot_id</span> = <span class="ruby-value str">'*'</span>
32:                         <span class="ruby-ivar">@rules</span> = []
33:                         <span class="ruby-ivar">@sitemaps</span> = []
34:                         <span class="ruby-ivar">@robot_id</span> = <span class="ruby-identifier">robot_id</span>.<span class="ruby-identifier">downcase</span> <span class="ruby-keyword kw">if</span> <span class="ruby-operator">!</span><span class="ruby-identifier">robot_id</span>.<span class="ruby-identifier">nil?</span>
35:                         
36:                 <span class="ruby-keyword kw">end</span>
</pre>
          </div>

        </div>
      </div>


      <h3 class="section-bar">Public Instance methods</h3>


      <div id="method-M000005" class="method-detail">
        <a name="M000005"></a>

        <div class="method-heading">

          <a href="#M000005" class="method-signature">

          <span class="method-name">allowed?</span><span class="method-args">(var)</span>

          </a>

        </div>

        <div class="method-description">

          <p>
Check if the <tt>URL</tt> is allowed to be crawled from the current
Robot_id.
</p>
<pre>
 client = Robotstxt::Robotstxtistance.new('my_robot_id')
 if client.get('http://www.simonerinzivillo.it')
   client.allowed?('http://www.simonerinzivillo.it/no-dir/')
 end
</pre>
<p>
This method returns <tt>true</tt> if the robots.txt file does not block the
access to the URL.
</p>

          <p><a class="source-toggle" href="#"
            onclick="toggleCode('M000005-source');return false;">[Source]</a></p>
          <div class="method-source-code" id="M000005-source">
<pre>
     <span class="ruby-comment cmt"># File lib/robotstxt/parser.rb, line 94</span>
 94:                 <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">allowed?</span>(<span class="ruby-identifier">var</span>)
 95:                         <span class="ruby-identifier">is_allow</span> = <span class="ruby-keyword kw">true</span>
 96:                         <span class="ruby-identifier">url</span> = <span class="ruby-constant">URI</span>.<span class="ruby-identifier">parse</span>(<span class="ruby-identifier">var</span>)
 97:                         <span class="ruby-identifier">querystring</span> = (<span class="ruby-operator">!</span><span class="ruby-identifier">url</span>.<span class="ruby-identifier">query</span>.<span class="ruby-identifier">nil?</span>) <span class="ruby-operator">?</span> <span class="ruby-value str">'?'</span> <span class="ruby-operator">+</span> <span class="ruby-identifier">url</span>.<span class="ruby-identifier">query</span> <span class="ruby-operator">:</span> <span class="ruby-value str">''</span>
 98:                         <span class="ruby-identifier">url_path</span> = <span class="ruby-identifier">url</span>.<span class="ruby-identifier">path</span> <span class="ruby-operator">+</span> <span class="ruby-identifier">querystring</span>
 99:                         
100:                         <span class="ruby-ivar">@rules</span>.<span class="ruby-identifier">each</span> {<span class="ruby-operator">|</span><span class="ruby-identifier">ua</span><span class="ruby-operator">|</span>
101:                                 
102:                                 <span class="ruby-keyword kw">if</span> <span class="ruby-ivar">@robot_id</span> <span class="ruby-operator">==</span> <span class="ruby-identifier">ua</span>[<span class="ruby-value">0</span>] <span class="ruby-operator">||</span> <span class="ruby-identifier">ua</span>[<span class="ruby-value">0</span>] <span class="ruby-operator">==</span> <span class="ruby-value str">'*'</span> 
103:                                         
104:                                         <span class="ruby-identifier">ua</span>[<span class="ruby-value">1</span>].<span class="ruby-identifier">each</span> {<span class="ruby-operator">|</span><span class="ruby-identifier">d</span><span class="ruby-operator">|</span>
105:                                                 
106:                                                 <span class="ruby-identifier">is_allow</span> = <span class="ruby-keyword kw">false</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">url_path</span>.<span class="ruby-identifier">match</span>(<span class="ruby-value str">'^'</span> <span class="ruby-operator">+</span> <span class="ruby-identifier">d</span> ) <span class="ruby-operator">||</span> <span class="ruby-identifier">d</span> <span class="ruby-operator">==</span> <span class="ruby-value str">'/'</span>
107:                                                 
108:                                         }
109:                                         
110:                                 <span class="ruby-keyword kw">end</span>
111:                                 
112:                         }
113:                         <span class="ruby-identifier">is_allow</span>
114:                 <span class="ruby-keyword kw">end</span>
</pre>
          </div>

        </div>
      </div>


      <div id="method-M000007" class="method-detail">
        <a name="M000007"></a>

        <div class="method-heading">

          <a href="#M000007" class="method-signature">

          <span class="method-name">found?</span><span class="method-args">()</span>

          </a>

        </div>

        <div class="method-description">

          <p>
This method returns <tt>true</tt> if the Robots.txt parsing is gone.
</p>

          <p><a class="source-toggle" href="#"
            onclick="toggleCode('M000007-source');return false;">[Source]</a></p>
          <div class="method-source-code" id="M000007-source">
<pre>
     <span class="ruby-comment cmt"># File lib/robotstxt/parser.rb, line 131</span>
131:                 <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">found?</span>
132:                         <span class="ruby-operator">!</span><span class="ruby-operator">!</span><span class="ruby-ivar">@found</span>
133:                 <span class="ruby-keyword kw">end</span>
</pre>
          </div>

        </div>
      </div>


      <div id="method-M000004" class="method-detail">
        <a name="M000004"></a>

        <div class="method-heading">

          <a href="#M000004" class="method-signature">

          <span class="method-name">get</span><span class="method-args">(hostname)</span>

          </a>

        </div>

        <div class="method-description">

          <p>
Requires and parses the Robots.txt file for the <tt>hostname</tt>.
</p>
<pre>
 client = Robotstxt::Robotstxtistance.new('my_robot_id')
 client.get('http://www.simonerinzivillo.it')
</pre>
<p>
This method returns <tt>true</tt> if the parsing is gone.
</p>

          <p><a class="source-toggle" href="#"
            onclick="toggleCode('M000004-source');return false;">[Source]</a></p>
          <div class="method-source-code" id="M000004-source">
<pre>
    <span class="ruby-comment cmt"># File lib/robotstxt/parser.rb, line 47</span>
47:                 <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">get</span>(<span class="ruby-identifier">hostname</span>)
48:                         
49:                         <span class="ruby-ivar">@ehttp</span> = <span class="ruby-keyword kw">true</span>
50:                         <span class="ruby-identifier">url</span> = <span class="ruby-constant">URI</span>.<span class="ruby-identifier">parse</span>(<span class="ruby-identifier">hostname</span>)
51:                         
52:                         <span class="ruby-keyword kw">begin</span>
53:                                 <span class="ruby-identifier">http</span> = <span class="ruby-constant">Net</span><span class="ruby-operator">::</span><span class="ruby-constant">HTTP</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">url</span>.<span class="ruby-identifier">host</span>, <span class="ruby-identifier">url</span>.<span class="ruby-identifier">port</span>)
54:                                 <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">url</span>.<span class="ruby-identifier">scheme</span> <span class="ruby-operator">==</span> <span class="ruby-value str">'https'</span>
55:                                         <span class="ruby-identifier">http</span>.<span class="ruby-identifier">verify_mode</span> = <span class="ruby-constant">OpenSSL</span><span class="ruby-operator">::</span><span class="ruby-constant">SSL</span><span class="ruby-operator">::</span><span class="ruby-constant">VERIFY_NONE</span>
56:                                         <span class="ruby-identifier">http</span>.<span class="ruby-identifier">use_ssl</span> = <span class="ruby-keyword kw">true</span> 
57:                                 <span class="ruby-keyword kw">end</span>
58:                                 
59:                                 <span class="ruby-identifier">response</span> =  <span class="ruby-identifier">http</span>.<span class="ruby-identifier">request</span>(<span class="ruby-constant">Net</span><span class="ruby-operator">::</span><span class="ruby-constant">HTTP</span><span class="ruby-operator">::</span><span class="ruby-constant">Get</span>.<span class="ruby-identifier">new</span>(<span class="ruby-value str">'/robots.txt'</span>))
60:                                 
61:                                 <span class="ruby-keyword kw">case</span> <span class="ruby-identifier">response</span>
62:                                         <span class="ruby-keyword kw">when</span> <span class="ruby-constant">Net</span><span class="ruby-operator">::</span><span class="ruby-constant">HTTPSuccess</span> <span class="ruby-keyword kw">then</span>
63:                                         <span class="ruby-ivar">@found</span> = <span class="ruby-keyword kw">true</span>
64:                                         <span class="ruby-ivar">@body</span> = <span class="ruby-identifier">response</span>.<span class="ruby-identifier">body</span>
65:                                         <span class="ruby-identifier">parse</span>()            
66:                                         
67:                                         <span class="ruby-keyword kw">else</span>
68:                                         <span class="ruby-ivar">@found</span> = <span class="ruby-keyword kw">false</span>
69:                                 <span class="ruby-keyword kw">end</span> 
70:                                 
71:                                 <span class="ruby-keyword kw">return</span> <span class="ruby-ivar">@found</span>
72:                                 
73:                                 <span class="ruby-keyword kw">rescue</span> <span class="ruby-constant">Timeout</span><span class="ruby-operator">::</span><span class="ruby-constant">Error</span>, <span class="ruby-constant">Errno</span><span class="ruby-operator">::</span><span class="ruby-constant">EINVAL</span>, <span class="ruby-constant">Errno</span><span class="ruby-operator">::</span><span class="ruby-constant">ECONNRESET</span> =<span class="ruby-operator">&gt;</span> <span class="ruby-identifier">e</span>
74:                                 <span class="ruby-keyword kw">if</span> <span class="ruby-ivar">@ehttp</span>
75:                                         <span class="ruby-ivar">@ettp</span> = <span class="ruby-keyword kw">false</span>
76:                                         <span class="ruby-keyword kw">retry</span> 
77:                                         <span class="ruby-keyword kw">else</span>
78:                                         <span class="ruby-keyword kw">return</span> <span class="ruby-keyword kw">nil</span>
79:                                 <span class="ruby-keyword kw">end</span>
80:                         <span class="ruby-keyword kw">end</span>
81:                         
82:                 <span class="ruby-keyword kw">end</span>
</pre>
          </div>

        </div>
      </div>


      <div id="method-M000006" class="method-detail">
        <a name="M000006"></a>

        <div class="method-heading">

          <a href="#M000006" class="method-signature">

          <span class="method-name">sitemaps</span><span class="method-args">()</span>

          </a>

        </div>

        <div class="method-description">

          <p>
Analyze the robots.txt file to return an <tt>Array</tt> containing the list
of XML Sitemaps URLs.
</p>
<pre>
 client = Robotstxt::Robotstxtistance.new('my_robot_id')
 if client.get('http://www.simonerinzivillo.it')
   client.sitemaps.each{ |url|
   puts url
 }
 end
</pre>

          <p><a class="source-toggle" href="#"
            onclick="toggleCode('M000006-source');return false;">[Source]</a></p>
          <div class="method-source-code" id="M000006-source">
<pre>
     <span class="ruby-comment cmt"># File lib/robotstxt/parser.rb, line 125</span>
125:                 <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">sitemaps</span>
126:                         <span class="ruby-ivar">@sitemaps</span>
127:                 <span class="ruby-keyword kw">end</span>
</pre>
          </div>

        </div>
      </div>



    </div>




  </div>

<div id="validator-badges">
  <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
</div>

</body>
</html>
